#!/user/bin
hdfs_path="viewfs://c9/user_ext/ad_engine/word2vec/location/sds_w2v_sample_followlist/dt=20190724/"
hdfs_output="viewfs://c9/user_ext/ad_engine/word2vec/location/sds_w2v_sample_user_vector/dt=20190731/"

local_path="/data0/vad/hdfs_word2vec/"
start_pos=0
end_pos=99

mkdir -p $local_path
cp head_vector.txt gen_vec $local_path
cd $local_path

while [ $start_pos -le  $end_pos ]
do
	
	file_name=""
	if [ $start_pos -lt 10 ]; then
		file_name="00000"$start_pos"_0"
	else
		file_name="0000"$start_pos"_0"
	fi

	hdfs_file=$hdfs_path$file_name	
	/data0/vad/anywhere_hdfs/hdfs.sh get $hdfs_file $local_path
	echo $hdfs_file
	
	#split
	num=`cat $local_path$file_name|wc -l`
	let num=num/10
	let num=num+1

	rm -rf div_*
	split -d -l $num $file_name	"div_"
	div_pos=0
	while [ $div_pos -lt 10 ]
	do
		div_file="div_0"$div_pos
		output_file=$local_path$file_name$div_pos
		./gen_vec ./head_vector.txt $div_file $file_name$div_pos &
		wait

		/data0/vad/anywhere_hdfs/hdfs.sh put $output_file $hdfs_output 
		echo "hdfs put "$output_file $hdfs_output
		rm -rf $output_file
		let div_pos=div_pos+1
	done

	rm -rf $file_name
	let start_pos=start_pos+1 
done

echo "end"
